package spark_study

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object WordCount {
  def main(args: Array[String]): Unit = {
    //创建sparkcontxt
    val conf: SparkConf = new SparkConf().setAppName("wordcount")
    //生成RDD
    val sc: SparkContext = new SparkContext(conf)
    //编写程序 对RDD进行编程
    //指定读取位置
    val line: RDD[String] = sc.textFile(args(0))
    //指定切分规则
    val words: RDD[String] = line.flatMap(_.split("\\s"))
    val wordAndOne: RDD[(String, Int)] = words.map((_, 1))
    val reduced: RDD[(String, Int)] = wordAndOne.reduceByKey(_ + _)
    val sorted: RDD[(String, Int)] = reduced.sortBy(_._2, false)
    sorted.saveAsTextFile(args(1))
    sc.stop()
  }
}
